{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Simple Character RNN \n",
    "# Code by GunhoChoi\n",
    "\n",
    "import torch \n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Preprocessing string data\n",
    "# alphabet(0-25), space(26), start(27), end(28) -> 29 chars (0-28)\n",
    "\n",
    "string = \"hello pytorch\"\n",
    "chars = \"abcdefghijklmnopqrstuvwxyz 01\"\n",
    "char_list = [i for i in chars]\n",
    "batch_size = len(char_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# String to onehot vector\n",
    "# a -> [1 0 0 ... 0 0]\n",
    "\n",
    "def string_to_onehot(string):\n",
    "    start = np.zeros(shape=len(char_list) ,dtype=int)\n",
    "    end = np.zeros(shape=len(char_list) ,dtype=int)\n",
    "    start[-2] = 1\n",
    "    end[-1] = 1\n",
    "    for i in string:\n",
    "        idx = char_list.index(i)\n",
    "        zero = np.zeros(shape=batch_size ,dtype=int)\n",
    "        zero[idx]=1\n",
    "        start = np.vstack([start,zero])\n",
    "    output = np.vstack([start,end])\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Onehot vector to word\n",
    "# [1 0 0 ... 0 0] -> a \n",
    "\n",
    "def onehot_to_word(onehot_1):\n",
    "    onehot = torch.Tensor.numpy(onehot_1)\n",
    "    return char_list[onehot.argmax()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# RNN with 1 hidden layer\n",
    "\n",
    "class RNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, output_size):\n",
    "        super(RNN, self).__init__()\n",
    "        \n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        \n",
    "        self.i2h = nn.Linear(input_size + hidden_size, hidden_size)\n",
    "        self.i2o = nn.Linear(input_size + hidden_size, output_size)\n",
    "    \n",
    "    def forward(self, input, hidden):\n",
    "        combined = torch.cat((input, hidden), 1)\n",
    "        hidden = self.i2h(combined)\n",
    "        output = self.i2o(combined)\n",
    "        return output, hidden\n",
    "\n",
    "    def init_hidden(self):\n",
    "        return Variable(torch.zeros(1, self.hidden_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Hyperparameters & Initialization of rnn\n",
    "\n",
    "n_letters = len(char_list)\n",
    "n_hidden = 100\n",
    "lr = 0.01\n",
    "epochs = 100\n",
    "\n",
    "one_hot = torch.from_numpy(string_to_onehot(string)).type_as(torch.FloatTensor())\n",
    "rnn = RNN(n_letters, n_hidden, n_letters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Loss function & Optimizer\n",
    "\n",
    "loss_func = nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(rnn.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 0.5952\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      "1.00000e-02 *\n",
      "  5.4592\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      "1.00000e-02 *\n",
      "  2.3330\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      "1.00000e-02 *\n",
      "  1.2092\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      "1.00000e-03 *\n",
      "  6.0728\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      "1.00000e-03 *\n",
      "  2.8333\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      "1.00000e-03 *\n",
      "  1.2252\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      "1.00000e-04 *\n",
      "  4.5199\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      "1.00000e-04 *\n",
      "  1.8941\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      "1.00000e-04 *\n",
      "  2.5393\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# train\n",
    "# total_loss, gradient, and hidden should be initialized in every epoch\n",
    "\n",
    "for i in range(epochs):\n",
    "    rnn.zero_grad()\n",
    "    total_loss = 0\n",
    "    hidden = rnn.init_hidden()\n",
    "\n",
    "    for j in range(one_hot.size()[0]-1):\n",
    "        input = Variable(one_hot[j:j+1,:])\n",
    "        output, hidden = rnn.forward(input, hidden)\n",
    "        target = Variable(one_hot[j+1])\n",
    "        loss = loss_func(output.view(-1),target.view(-1))\n",
    "        total_loss += loss\n",
    "        input = output\n",
    "\n",
    "    total_loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    if i % 10 == 0:\n",
    "        print(total_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "h\n",
      "e\n",
      "l\n",
      "l\n",
      "o\n",
      " \n",
      "p\n",
      "y\n",
      "t\n",
      "o\n",
      "r\n",
      "c\n",
      "h\n"
     ]
    }
   ],
   "source": [
    "# test \n",
    "# hidden state should be initialized once\n",
    "\n",
    "hidden = rnn.init_hidden()\n",
    "input = Variable(one_hot[0:1,:])\n",
    "\n",
    "for i in range(len(string)):\n",
    "    output, hidden = rnn.forward(input, hidden)\n",
    "    print(onehot_to_word(output.data))\n",
    "    input = output"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
